package ca.uwindsor.cs.deepweb.utility.gov2;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Date;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.LockObtainFailedException;

import ca.uwindsor.cs.deepweb.estimation.FilesContentIndexer;
import ca.uwindsor.cs.deepweb.utility.NekoHTMLHandler;

public class DocumentSizeIndexerforGOV2 {

	private int globalID;

	protected IndexWriter indexWriter;

	/**
	 * Constructor
	 */
	public DocumentSizeIndexerforGOV2() {
		globalID=1;
	}

	/**
	 * Start to create the index to store file size
	 * 
	 * @param indexdir
	 *            Where to store the index
	 * @param docdir
	 *            Where are the documents locate
	 */
	public void index(String indexdir, File docdir) {
		try {
			Analyzer luceneAnalyzer = new StandardAnalyzer();
			long startTime = new Date().getTime();
			indexWriter = new IndexWriter(indexdir, luceneAnalyzer, true);
			index(docdir);

			long endTime = new Date().getTime();
			System.out
					.println("It takes "
							+ (endTime - startTime)
							+ " milliseconds to create index for the files in directory "
							+ docdir.getPath()
							+ "\nTotal documnents indexed is:" + (globalID - 1));

			System.out
					.println("index is completed! \n Start to optimize the index...");
			indexWriter.optimize();
			System.out.println("optimization is completed!");
			indexWriter.close();
		} catch (CorruptIndexException e) {
			e.printStackTrace();
			System.exit(1);
		} catch (LockObtainFailedException e) {
			e.printStackTrace();
			System.exit(1);
		} catch (IOException e) {
			e.printStackTrace();
			System.exit(1);
		}

	}

	protected void index(File dir) throws CorruptIndexException,
			LockObtainFailedException, IOException {
		File[] dataFiles = dir.listFiles();

		// Add documents to the index
		for (int i = 0; i < dataFiles.length; i++) {
			File file = dataFiles[i];
			if (file.isFile()) {
				if (file.length() > 2048) {
					NekoHTMLHandler handler = new NekoHTMLHandler("latin1");
					try {
						handler
								.processContent(new FileInputStream(
										dataFiles[i]));
					} catch (Exception e) {
						// e.printStackTrace();
						System.err.println("Error on parsing: "
								+ dataFiles[i].getCanonicalPath());
						continue;
					}
					String content = handler.getContent().replaceAll("\\s+",
							" ");
					if (content.length() < 2048) {
						continue;
					}

					System.out.println("Indexing file "
							+ dataFiles[i].getCanonicalPath());

					Document document = new Document();
					// make the ID from the path shorter - get the file name
					String filepath = file.getCanonicalPath();
					int offset1 = filepath.lastIndexOf(java.io.File.separator);
					int offset2 = filepath.lastIndexOf(".");
					String word = new String(filepath.substring(offset1 + 1,
							offset2));

					// String word = String.valueOf(globalID);
					// System.out.println(id);

					Field FieldPath = new Field(FilesContentIndexer.FIELD_ID,
							word, Field.Store.YES, Field.Index.UN_TOKENIZED);

					Field FieldSize = new Field(FilesContentIndexer.FIELD_SIZE,
							Integer.toString(content.length()),
							Field.Store.YES, Field.Index.UN_TOKENIZED);

					document.add(FieldPath);
					document.add(FieldSize);
					indexWriter.addDocument(document);
					// txtReader.close();
					globalID++;
				}
			} else if (file.isDirectory()) {
				index(dataFiles[i]);
			}
		}

	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		DocumentSizeIndexerforGOV2 index = new DocumentSizeIndexerforGOV2();
		if(args.length==2){
		index.index(args[1], new File(args[0]));
		}else{
			System.out.println("usage: java -jar DocumentSizeIndexerforGOV2 documentDir indexDir");
		}
	}

}
